# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import json, redis, pickle
import logging

logger = logging.getLogger(__name__)

class DoubanPipeline(object):
    def __init__(self):
        redis_pool = redis.ConnectionPool(host='127.0.0.1', port=6379, db=0, password='Schenker123')
        # redis_pool = redis.ConnectionPool(host='127.0.0.1', port=6379, db=0)
        self.server = redis.Redis(connection_pool=redis_pool)
        logger.info('===============Pipeline begin===============')
        tag_name = self.server.get('tag_name')
        if tag_name:
            output_file_name = '%s.json' %str(tag_name)
        else:
            output_file_name = 'douban_book.json'
        self.f = open(output_file_name, 'a')

    def process_item(self, item, spider):
        valid = True
        for data in item:
            if not data:
                valid = False
                raise DropItem("Missing {0}!".format(data))
        if valid:
            try:
                # This returns the number of values added, zero if already exists.
                key = item['sku']
                item_name = item['name']
                # if self.server.get(key) is not None:
                book_url_exist = self.server.sismember("book_url_set",  item['book_url'])
                page_cnt_process = self.server.get("page_cnt_process")
                if page_cnt_process:
                    page_cnt_process = int(page_cnt_process)
                else:
                    page_cnt_process = 0
                if book_url_exist == 0:
                    self.server.sadd("book_url_set", item['book_url'])
                    json.dump(item, self.f, ensure_ascii=False)
                    self.f.write(',\n')
                    try:
                        if page_cnt_process == 5:
                            self.f.flush()
                        else:
                            pass
                    except Exception as e:
                        logger.error(e)
                # if not self.server.hexists(name='douban_book', key=key):
                #     self.server.hset(name='douban_book', key=key, value=item_name)
                #     self.server.sadd("book_url_set", item['book_url'])
                #     # self.server.set(key, item_name)
                #     json.dump(item, self.f, ensure_ascii=False)
                #     self.f.write(',\n')
                    logger.info("<<<Successfully saved {} | {} | {}>>>".format(item['sku'], item['name'], item.get('isbn','')))
            except Exception as err:
                logger.error(err)
                # raise DropItem("Duplicated Item: {}".format(item['name']))
        return item


    def close_spider(self, spider):
        self.f.close()
        logger.info('================Pipeline end================')